pathBase = 'dataset/chat_corpus/clean_chat_corpus/'
paths = [
    "chatterbot.tsv",
    "douban_single_turn.tsv",
    "ptt.tsv",
    "qingyun.tsv",
    "subtitle.tsv",
    "tieba.tsv",
    "weibo.tsv"
]
fullPaths = [pathBase + path for path in paths]

from src import PreproductionByFrequency as Pre

def gen():
    id, ch = Pre.genHanzi2OneHot(fullPaths)
    with open("results/hanzi_2_one_hot_freq.data", "w") as f:
        for item in ch:
            f.write(item + '\n')

gen()